#!/usr/bin/python
# -*- coding: utf-8-*-
import urllib
import sgmllib
import time
import logging
import datetime
from google.appengine.ext import db

begin_label = u'可售期房统计'
char_in_date='0123456789-'

seq = [
       [u'期房网上签约', u'住宅套数', u'面积'],
       [u'现房网上签约', u'住宅套数', u'面积'],
       [u'存量房网上签约', u'住宅签约套数', u'住宅签约面积'],
       [u'存量房网上签约', u'住宅签约套数', u'住宅签约面积']
      ] 


class tdparser(sgmllib.SGMLParser):
    def reset(self):
        sgmllib.SGMLParser.reset(self)
        self.content_in_td=[]
        self.data_in_td = 0
        self.tmp_str = u''
        self.begin = False
    def handle_data(self, attrs):
        if self.begin:
            if self.data_in_td:
                self.tmp_str += attrs
        else:
            # !FIXME! Because gae running with python 2.5, illegal str byte \xb7 will lead to UnicodeDecodeError
            try :
                if attrs.find(begin_label) >= 0:
                    self.begin = True
                    #print 'I got begin label'
            except:
                pass
    def start_td(self, attrs):
        self.data_in_td = 1;
        self.tmp_str = u''
    def end_td(self):
        self.data_in_td = 0;
        self.content_in_td.append(self.tmp_str)

def get_data(input_url):
    request_url = input_url
    need_retry = 1;
    retry_cnt = 0;
    while(need_retry):
        try:
            feeddata=urllib.urlopen(request_url).read().decode('gb2312');
            need_retry = 0;
            retry_cnt = 0;
        except:
            time.sleep(5*retry_cnt)
            if retry_cnt < 10:
                logging.error('open url failed, retrying...')
                need_retry = 1;
                retry_cnt += 1;
            else:
                logging.error('open url failed 10 times, give up retry')
                need_retry = 0;
                retry_cnt = 0;
                feeddata = ''
                return


    myparser=tdparser()
    myparser.feed(feeddata)

    line_number = 0
    column_number = 0

    immediate = 0
    value = [
            ['']*4, #[期房网上签约住宅，套数，面积，日期]
            ['']*4, #[现房网上签约住宅，套数，面积，日期]
            ['']*4,
            ['']*4  #[存量房网上签约住，套数，面积，日期]
    ]
    for i in myparser.content_in_td:
        if immediate:
            #print ' line is %r, columns is %r, I got %r'%(line_number, column_number, i)
            value[line_number][column_number] = i
            immediate = 0
            if column_number==2:   #each line has 3 columns, if column_number equal 2, the last column has reached, go on to next line
                column_number=0
                line_number +=1
            else:
                column_number += 1
        if line_number < len(seq):
            #print ' line is %d, column is %d, looking for %r in %r'%(line_number, column_number, seq[line_number][column_number], i)
            if i.find(seq[line_number][column_number]) >= 0:
                immediate = 1
                if column_number==0 and (line_number in [0,1,3]):
                    got_date=''
                    for j in i:
                        if j in char_in_date:
                            got_date += j
                    #print 'I got date%r'%got_date
                    value[line_number][3] = got_date
    return value

def update_database():
    url = 'http://www.bjfdc.gov.cn/public/Index.asp'
    retry = True
    cnt = 0
    while(retry):
        try:
            value = get_data(url)
            retry = False
        except:
            logging.error('google.appengine.runtime.DeadlineExceededError\n')
            if cnt > 10000:
                retry = False
            cnt += 1
            pass 
    record_query = Datalog.all().order('-date')

    y,m,d=value[0][3].split('-')
    now = datetime.datetime(int(y),int(m),int(d))
    records = record_query.fetch(1) 
    if records:
        y,m,d=records[0].disp_date.split('-')
        last = datetime.datetime(int(y),int(m),int(d))
    else:
        last = datetime.datetime(1982,1,1)
    
    if now > last:
        record = Datalog(
            qfts=int(value[0][1]),
            qfmj=int(value[0][2]),
            xfts=int(value[1][1]),
            xfmj=int(value[1][2]),
            clfts=int(value[3][1]),
            clfmj=int(value[3][2]),
            disp_date = value[0][3]
        )
        record.put()
    else:
        #print 'Got is %r, lt last %r'%(value[0][3], records[0].disp_date)
        pass
    return

class Datalog(db.Model):
    date = db.DateTimeProperty(auto_now_add=True)
    qfts = db.IntegerProperty()
    qfmj = db.IntegerProperty()
    xfts = db.IntegerProperty()
    xfmj = db.IntegerProperty()
    clfts = db.IntegerProperty()
    clfmj = db.IntegerProperty()
    disp_date = db.StringProperty()

if __name__=='__main__':
    update_database()
